
import bs4
import re
import os
import pandas as pd
from cleaner import HTMLCleaner
from util import FileUtil, TreeUtil
from extrator.TPR.TPR import NodePath

file_list = list(FileUtil.all_files('../output'))

#time_pattern = '(([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))-02-29)'
time_pattern = ['\d{2,4}-\d{1,2}-\d{1,2}','\d{2,4}年\d{1,2}月\d{1,2}','\d{1,2}-\d{1,2}']

# 删除已有的.after文件
for p in FileUtil.all_files('../output', '*.after'):
    os.remove(p)

# file_path = '../output/18'

for file_path in file_list:
    print(file_path)
    with open(file_path, 'rt', encoding='utf-8') as f:
        html = f.read()

    dom = bs4.BeautifulSoup(html, 'lxml')
    HTMLCleaner.clean(dom)

    print('find node')
    time_node_list = []
    for pattern in time_pattern:
        for n in dom.find_all(text=re.compile(pattern)):
            time_node_list.append(n)
        
    time_node_path_dic = {}

    def get_path(node):
        path = []
        while node.parent is not None:
            node = node.parent
            path.append(node.name)
        return str(path)

    for node in time_node_list:
        path = get_path(node)
        nodePath = time_node_path_dic.get(path)
        if nodePath is not None:
            nodePath.pathNum += 1
        else:
            nodePath = NodePath(path)
            nodePath.pathNum = 1
            time_node_path_dic[path] = nodePath
        nodePath.append(node)

    data = pd.DataFrame()
    data['path'] = time_node_path_dic.keys()
    data['pathNode'] = [time_node_path_dic[p] for p in data.path]
    data['count'] = [n.pathNum for n in data.pathNode]
    if len(data) == 0:
        continue
    data = data.sort_values('count', ascending=False)
    data.index = range(len(data))
    time_nodes = data.loc[0].pathNode.nodes
    if len(time_nodes) == 0 :
        continue
    parent = TreeUtil.cal_parent(time_nodes[0], time_nodes[-1])

    with open(file_path+'.after', 'wt', encoding='utf-8') as f:
        print('write %s' % f.name)
        f.write(str(parent))
